#!/usr/bin/python
# This script scans a text file and creates tags of the most used words.
# The script has been released under BSD license. Copyright (C) 2010 \Reiner Rottmann <rottmannATTrottmann..it>

import string 
import re
from nltk.corpus import stopwords

def sort_by_value(d): 
    u""" Returns the keys of dictionary d sorted by their values """ 
    items=d.items() 
    backitems=[ [v[1],v[0]] for v in items] 
    backitems.sort() 
    return [ backitems[i][1] for i in range(0,len(backitems))] 

def get_most_used_words(text, n):
    """ Returns the top n used words in a text without counting the top 100 most \
used words in English and German language"""
    
    # only print lower case words
    text =  text.lower()
    
    
    # replace seperators with spaces
    seperators = "\n\r\f\t\v.,/\\""''"
    #seperators = "\n\r.,"
    for seperator in seperators:
        #text = text.replace(seperator, " ")
        text = text.replace(seperator, " ")
    
    # remove invalid chars
    validchars = "abcdefghijklmnopqrstuvwxyz "
    
    charlist = list(text.lower())
    charlist = [char for char in charlist if char in validchars]
    text = "".join(charlist)
    
    # split up the words
    words  = text.split(" ")
    
    # remove empty list words
    words = [word for word in words if word != ""]
    
    # sort words
    words.sort()
    
   
   
    
    # be picky what to include in the word list
    sorted = {}
    for word in words:
        # accept only words with lenght between 3 and 15
        if len(word) > 3 and len(word) < 15:
            # accept only words that are not in blacklists of the top 100 common 
	   if word not in stopwords.words('english'):
                    sorted[word] = sorted.get(word,0) + 1
   
    # use only the top n words
    top =  sort_by_value(sorted)[-n:]
    
    # don't use the number of occurances as sort criteria anymore
    top.sort()
        
    return " ".join(top)

def usage():
    """ This function prints the usage information"""
    import sys
    print """This script has been released under BSD license. Copyright (C) 2010 Reiner Rottmann <rottmannATrottmann.it>"""

def main():
    """ This is the main function"""
    import sys
    # check whether no commandline argument has been entered and show usage
    if len(sys.argv) == 1:
        usage()
    else:
        # read from stdin if "-" is present as first commandline argument
        if sys.argv[1] == "-":
            text = sys.stdin.read()
        else:
            # open file in any other case; should there be errors, print error message and usage
            try:    
                file = sys.argv[1]
                fd = open(file, 'r')
                text = fd.read()
                fd.close()
            except:
                print "ERROR: Input file could not be opened."
                usage()
    # get top 25 used words
    print get_most_used_words(text,10)

# run main function if called directly
if __name__=="__main__":
   main() 
 



